{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "override USE_NET <class 'src.networks.XintongNetwork'>\n",
      "override DATASET_PROC_METHOD_TRAIN Random\n",
      "override DATASET_PROC_METHOD_VAL Rescale\n",
      "override MAX_CATEGORY_NUM 1\n",
      "override IMAGE_EMBED_SIZE 512\n",
      "override WEIGHT_IMAGE_TEXT 1.0\n",
      "override F_WEIGHT_SPARSE_SOFTMAX 1.0\n",
      "override B_WEIGHT_SPARSE_SOFTMAX 1.0\n",
      "override USE_PRETRAINED_WORD_EMBEDDING True\n",
      "override WORD_EMBED_SIZE 300\n",
      "override MAX_VOCAB_SIZE 2500\n",
      "override OUTFIT_NAME_PAD_NUM 10\n",
      "override NUM_EPOCH 20\n",
      "override LEARNING_RATE 0.2\n",
      "override LEARNING_RATE_DECAY 0.5\n",
      "override LEARNING_RATE_DECAY_EVERY_EPOCHS 2\n",
      "override GRADIENT_CLIP 5\n",
      "override BATCH_SIZE 10\n",
      "override SAVE_EVERY_STEPS 10000\n",
      "override SAVE_EVERY_EPOCHS 1\n",
      "override VAL_WHILE_TRAIN True\n",
      "override VAL_FASHION_COMP_FILE fashion_compatibility_small.txt\n",
      "override VAL_FITB_FILE fill_in_blank_test_small.json\n",
      "override VAL_BATCH_SIZE 8\n",
      "override VAL_EVERY_STEPS 500\n",
      "override VAL_EVERY_EPOCHS 1\n",
      "override VAL_START_EPOCH 1\n",
      "override device cuda:0\n",
      "override TRAIN_DIR runs/src.conf.xintong/11-13 21:03:16\n",
      "override VAL_DIR runs/src.conf.xintong/11-13 21:03:16\n",
      "override MODEL_NAME src.conf.xintong\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.utils.data\n",
    "from src.const import base_path\n",
    "import numpy as np\n",
    "import cv2\n",
    "from torchvision import transforms\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "from skimage import io, transform\n",
    "import skimage\n",
    "from src import const\n",
    "import json\n",
    "import os\n",
    "import nltk\n",
    "from src.utils import load_json, build_vocab, Vocab\n",
    "from src.dataset import *\n",
    "from src.base_networks import *\n",
    "from src.networks import *\n",
    "from torch import nn\n",
    "import torchvision\n",
    "from torch.nn import functional as F\n",
    "from src.utils import merge_const\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from src.utils import FITBBenchMarkHelper\n",
    "merge_const('src.conf.xintong')\n",
    "const.BATCH_SIZE = 3\n",
    "# const.device = 'cpu'\n",
    "class _(object):\n",
    "    pass\n",
    "self = _()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fashion Compatibility Test, Use File: /home/hzy/datasets/polyvore/fashion_compatibility_small.txt\n",
      "FITB Test, Use File: /home/hzy/datasets/polyvore/train_fitb_test.json\n",
      "dict_keys(['raw_images', 'images', 'image_mask', 'item_nums', 'word_ids', 'word_mask', 'word_lengths', 'word_detail_mask', 'word_embedding_divider', 'types', 'b_images', 'b_image_mask', 'b_types'])\n"
     ]
    }
   ],
   "source": [
    "train_set = load_json(os.path.join(const.base_path, 'train_no_dup.json'))\n",
    "test_set = load_json(os.path.join(const.base_path, 'test_no_dup.json'))\n",
    "comp_dataset = CompatibilityBenchmarkDataset(const.DATASET_PROC_METHOD_VAL, test_set)\n",
    "const.VAL_FITB_FILE = 'train_fitb_test.json'\n",
    "fitb_dataset = FITBBenchmarkDataset(const.DATASET_PROC_METHOD_VAL, train_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hzy/anaconda3/lib/python3.6/site-packages/torch/nn/modules/rnn.py:38: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.7 and num_layers=1\n",
      "  \"num_layers={}\".format(dropout, num_layers))\n"
     ]
    }
   ],
   "source": [
    "net = const.USE_NET()\n",
    "net.load_state_dict(torch.load('./models/src.conf.xintong/model.pt-epoch19'))\n",
    "net = net.to(const.device)\n",
    "net.eval()\n",
    "self = net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "def flip(x, dim):\n",
    "    dim = x.dim() + dim if dim < 0 else dim\n",
    "    inds = tuple(slice(None, None) if i != dim\n",
    "                 else x.new(torch.arange(x.size(i) - 1, -1, -1).tolist()).long()\n",
    "                 for i in range(x.dim()))\n",
    "    return x[inds]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([-15.2119,  -3.2539, -11.8738, -25.6378], device='cuda:0') 1\n",
      "tensor([-33.7537, -13.7653, -19.8933,  -3.5922], device='cuda:0') 3\n",
      "tensor([ -6.9554, -22.2994, -23.7393, -13.3976], device='cuda:0') 0\n",
      "tensor([ -0.5496, -22.7796, -18.6699,  -6.8044], device='cuda:0') 0\n",
      "tensor([-32.9155, -16.1014,  -4.1965,  -4.0605], device='cuda:0') 3\n",
      "tensor([-10.5712, -12.6908,  -9.2035,  -0.3044], device='cuda:0') 3\n",
      "tensor([-15.9450, -11.8631, -34.2012, -30.0518], device='cuda:0') 1\n",
      "Val FITB step [10/300] accuracy: 0.2222222222222222\n",
      "forward 1\n",
      "forward 0\n",
      "tensor([-35.3496,  -5.4112, -25.3575, -44.3691], device='cuda:0') 1\n",
      "forward 2\n",
      "tensor([-14.1250, -16.9113, -20.6555,  -8.9163], device='cuda:0') 3\n",
      "tensor([-14.4376, -31.1132, -29.7282, -24.0870], device='cuda:0') 0\n",
      "tensor([ -9.3057, -13.3447,  -9.8115,  -7.3148], device='cuda:0') 3\n",
      "Val FITB step [20/300] accuracy: 0.3157894736842105\n",
      "tensor([-15.2407, -18.0105, -30.6744, -13.4480], device='cuda:0') 3\n",
      "tensor([-15.9357,  -0.1567,  -9.1990, -14.0721], device='cuda:0') 1\n",
      "tensor([-18.5354,  -0.0193, -32.9883, -18.7146], device='cuda:0') 1\n",
      "tensor([ -4.0778, -16.2971,  -6.9403,  -1.3413], device='cuda:0') 3\n",
      "tensor([-5.0545e-04, -3.7051e+01, -4.2417e+01, -2.5009e+01], device='cuda:0') 0\n",
      "tensor([ -5.4938, -10.3420, -25.5445,  -4.9478], device='cuda:0') 3\n",
      "forward 1\n",
      "tensor([-14.9486, -26.1504,  -0.5103, -19.8305], device='cuda:0') 2\n",
      "forward 1\n",
      "Val FITB step [30/300] accuracy: 0.2413793103448276\n",
      "tensor([ -9.7820,  -0.3115, -12.2504,  -8.3837], device='cuda:0') 1\n",
      "tensor([ -9.2769,  -2.0363, -34.5297, -25.2520], device='cuda:0') 1\n",
      "tensor([ -1.0079,  -2.0012, -18.9111, -24.1030], device='cuda:0') 0\n",
      "tensor([-34.7089, -15.8412, -57.2821,  -0.2613], device='cuda:0') 3\n",
      "tensor([-13.9317, -12.8670, -16.0746, -13.4435], device='cuda:0') 1\n",
      "tensor([ -5.9047, -20.9583, -19.4181, -27.7935], device='cuda:0') 0\n",
      "tensor([-34.7480,  -0.0485, -21.7664, -10.7039], device='cuda:0') 1\n",
      "tensor([-21.6664, -14.3611,  -5.9677,  -7.9737], device='cuda:0') 2\n",
      "Val FITB step [40/300] accuracy: 0.23076923076923078\n",
      "tensor([-20.6423, -14.3167, -13.2049, -16.6868], device='cuda:0') 2\n",
      "tensor([ -4.6633, -35.0539, -25.4193, -22.5976], device='cuda:0') 0\n",
      "tensor([  0.0000, -52.3556, -36.2147, -59.1973], device='cuda:0') 0\n",
      "tensor([-43.5492, -44.4882,  -0.0020, -20.2726], device='cuda:0') 2\n",
      "forward 0\n",
      "tensor([-22.4677,  -2.3558, -26.4789, -15.2571], device='cuda:0') 1\n",
      "tensor([-19.9451, -25.1848,  -6.0809, -38.9852], device='cuda:0') 2\n",
      "tensor([-22.8376, -15.9539,  -9.4730,  -9.3635], device='cuda:0') 3\n",
      "tensor([-22.0110,  -1.7512, -22.2032, -13.9570], device='cuda:0') 1\n",
      "forward 2\n",
      "Val FITB step [50/300] accuracy: 0.24489795918367346\n",
      "tensor([-33.0101, -23.0809, -32.4810,  -6.2141], device='cuda:0') 3\n",
      "tensor([ -5.1314, -15.6371, -12.3858,  -9.7992], device='cuda:0') 0\n",
      "tensor([-45.8640, -44.5359,  -0.0010, -29.4400], device='cuda:0') 2\n",
      "tensor([ -0.0316, -34.4914, -21.1964, -11.6224], device='cuda:0') 0\n",
      "tensor([-16.3693,  -1.6079, -17.5087, -11.9584], device='cuda:0') 1\n",
      "tensor([-18.5847, -14.1373,  -5.0303, -14.8465], device='cuda:0') 2\n",
      "tensor([-23.1113,  -0.0022, -18.1655, -46.5232], device='cuda:0') 1\n",
      "tensor([-17.7420, -20.5416, -24.2064, -33.8781], device='cuda:0') 0\n",
      "forward 1\n",
      "Val FITB step [60/300] accuracy: 0.2711864406779661\n",
      "tensor([-27.8868, -11.2896, -20.7646, -23.3196], device='cuda:0') 1\n",
      "tensor([-3.6240e-04, -3.8956e+01, -4.4391e+01, -3.1834e+01], device='cuda:0') 0\n",
      "tensor([-24.2327, -56.9262,  -1.5880,  -1.2109], device='cuda:0') 3\n",
      "tensor([-21.3059, -14.7090,  -9.6127,  -3.7789], device='cuda:0') 3\n",
      "forward 0\n",
      "tensor([-15.5987,  -0.0473, -45.2122, -15.3629], device='cuda:0') 1\n",
      "tensor([ -0.6351, -23.9207, -15.1367, -36.0107], device='cuda:0') 0\n",
      "tensor([-16.8062, -12.2626, -16.0737,  -5.1266], device='cuda:0') 3\n",
      "tensor([-26.3463, -12.6338,  -0.0039, -45.3590], device='cuda:0') 2\n",
      "Val FITB step [70/300] accuracy: 0.2898550724637681\n",
      "forward 3\n",
      "tensor([-30.6933, -21.9541, -10.0155, -28.6157], device='cuda:0') 2\n",
      "tensor([-11.8782, -25.4466, -15.8725,  -6.0336], device='cuda:0') 3\n",
      "forward 3\n",
      "tensor([-2.0608e+01, -2.1921e+01, -4.0964e+01, -2.2316e-04], device='cuda:0') 3\n",
      "tensor([-28.3222, -47.8318, -20.1510, -65.7574], device='cuda:0') 2\n",
      "tensor([-13.8337, -11.0572, -22.7510, -24.4836], device='cuda:0') 1\n",
      "tensor([-66.7167, -12.8005, -30.5502, -19.9301], device='cuda:0') 1\n",
      "Val FITB step [80/300] accuracy: 0.26582278481012656\n",
      "tensor([-15.9399,  -1.2105, -15.6096, -15.0482], device='cuda:0') 1\n",
      "tensor([-79.6681,  -5.0480,  -0.1889, -33.4053], device='cuda:0') 2\n",
      "tensor([-23.8515,  -0.4989, -49.8412,  -3.1004], device='cuda:0') 1\n",
      "forward 2\n",
      "tensor([ -2.8468, -25.2453, -13.4880, -19.2867], device='cuda:0') 0\n",
      "tensor([-27.7496, -11.9161, -45.2890, -13.0620], device='cuda:0') 1\n",
      "forward 2\n",
      "tensor([-15.1713,  -0.0076, -39.1019, -21.5432], device='cuda:0') 1\n",
      "tensor([-23.3264, -31.4686,  -0.0262, -32.4971], device='cuda:0') 2\n",
      "Val FITB step [90/300] accuracy: 0.24719101123595505\n",
      "tensor([-40.7336, -16.9106, -10.2125, -29.4975], device='cuda:0') 2\n",
      "tensor([-27.6944, -22.5071,  -0.1251,  -8.4777], device='cuda:0') 2\n",
      "forward 0\n",
      "forward 3\n",
      "tensor([-10.6950, -11.5597, -11.4406, -15.6385], device='cuda:0') 0\n",
      "tensor([ -6.8348, -16.7831, -35.7401, -21.9151], device='cuda:0') 0\n",
      "tensor([-28.7045, -17.7665,  -2.6042, -50.0360], device='cuda:0') 2\n",
      "tensor([-2.8038e-04, -4.1410e+01, -1.7928e+01, -4.6839e+01], device='cuda:0') 0\n",
      "tensor([-20.3167,  -9.2022, -18.2973, -20.0690], device='cuda:0') 1\n",
      "tensor([-19.9207,  -8.0114, -40.3406, -41.6800], device='cuda:0') 1\n",
      "Val FITB step [100/300] accuracy: 0.26262626262626265\n",
      "tensor([-22.7681, -44.6049, -10.0753,  -4.1994], device='cuda:0') 3\n",
      "forward 2\n",
      "tensor([-47.4230, -13.2495, -49.9739,  -8.3046], device='cuda:0') 3\n",
      "forward 1\n",
      "forward 2\n",
      "tensor([-17.1311,  -1.6182, -25.1539, -20.8700], device='cuda:0') 1\n",
      "forward 0\n",
      "tensor([ -6.3322, -16.2223, -48.9001,  -5.2638], device='cuda:0') 3\n",
      "tensor([-4.7684e-07, -7.3106e+01, -6.4693e+01, -8.5777e+01], device='cuda:0') 0\n",
      "Val FITB step [110/300] accuracy: 0.25688073394495414\n",
      "tensor([-2.3842e-07, -4.6987e+01, -5.4597e+01, -7.1995e+01], device='cuda:0') 0\n",
      "tensor([ -0.0054, -17.8901, -29.3407, -27.6428], device='cuda:0') 0\n",
      "tensor([-4.1449e+01, -9.3079e-04, -4.9738e+01, -2.7766e+01], device='cuda:0') 1\n",
      "tensor([-15.5763, -14.0271, -32.8894, -29.8229], device='cuda:0') 1\n",
      "forward 1\n",
      "forward 0\n",
      "Val FITB step [120/300] accuracy: 0.2605042016806723\n",
      "tensor([-11.3189,  -8.1009, -18.8096, -19.7454], device='cuda:0') 1\n",
      "tensor([-39.4030, -45.5694, -33.9889, -44.7924], device='cuda:0') 2\n",
      "tensor([ -0.0803, -35.1922, -48.4387,  -8.3346], device='cuda:0') 0\n",
      "tensor([-34.9120, -30.1294, -62.4572, -21.0392], device='cuda:0') 3\n",
      "tensor([-34.8827, -22.0417, -15.9211,  -1.5407], device='cuda:0') 3\n",
      "tensor([-16.0628,  -7.0291, -19.7379, -11.3323], device='cuda:0') 1\n",
      "tensor([ -0.3194, -35.8074, -26.8545, -11.5632], device='cuda:0') 0\n",
      "tensor([ -7.1492, -32.0230,  -8.9266, -24.7996], device='cuda:0') 0\n",
      "Val FITB step [130/300] accuracy: 0.26356589147286824\n",
      "tensor([-27.5635,  -2.8553,  -9.5849, -20.0925], device='cuda:0') 1\n",
      "tensor([ -3.6614, -20.4023, -13.4658, -15.4742], device='cuda:0') 0\n",
      "tensor([-55.7445, -22.2410, -26.2190, -33.0941], device='cuda:0') 1\n",
      "tensor([-12.7085,  -9.8320, -13.1920, -36.5192], device='cuda:0') 1\n",
      "tensor([-2.7075e+01, -2.2377e+01, -2.0852e+01, -1.2398e-04], device='cuda:0') 3\n",
      "tensor([-10.0148,  -5.8199, -10.9816, -68.4151], device='cuda:0') 1\n",
      "tensor([ -4.1766,  -6.4866, -85.3664,  -1.9859], device='cuda:0') 3\n",
      "tensor([-12.7142,  -3.6549,  -6.8239, -11.0012], device='cuda:0') 1\n",
      "Val FITB step [140/300] accuracy: 0.2589928057553957\n",
      "tensor([-40.6446, -12.0473,  -0.8860, -50.6219], device='cuda:0') 2\n",
      "forward 1\n",
      "tensor([-26.5145,  -5.6912, -28.1810,  -3.0563], device='cuda:0') 3\n",
      "forward 3\n",
      "tensor([-27.8180, -12.5620, -10.3211, -12.3444], device='cuda:0') 2\n",
      "tensor([ -6.0838,  -2.9292, -12.0252,  -9.0733], device='cuda:0') 1\n",
      "tensor([-13.6952,  -0.8941, -17.1629, -14.8165], device='cuda:0') 1\n",
      "tensor([-27.4012, -22.4108,  -1.9638, -31.5857], device='cuda:0') 2\n",
      "tensor([ -1.9136, -15.2553,  -7.5795, -35.5639], device='cuda:0') 0\n",
      "Val FITB step [150/300] accuracy: 0.2550335570469799\n",
      "forward 3\n",
      "tensor([ -1.2328, -35.4624, -24.3484, -24.9451], device='cuda:0') 0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([-11.2131, -10.2682, -48.3647, -13.7184], device='cuda:0') 1\n",
      "tensor([-37.4195,  -1.9299, -40.0498, -36.2016], device='cuda:0') 1\n",
      "tensor([-20.6675, -19.7163, -13.6472, -27.2494], device='cuda:0') 2\n",
      "tensor([ -4.0020, -23.0888,  -4.9031, -17.5372], device='cuda:0') 0\n",
      "tensor([ -6.8179, -26.8978,  -1.3073,  -5.1128], device='cuda:0') 2\n",
      "forward 0\n",
      "tensor([ -4.5700,  -3.7886, -14.7171, -20.2094], device='cuda:0') 1\n",
      "tensor([-3.4876e+01, -6.2532e+01, -1.9073e-06, -5.4265e+01], device='cuda:0') 2\n",
      "Val FITB step [160/300] accuracy: 0.2578616352201258\n",
      "tensor([ -0.4195, -29.1628, -11.6276, -20.6486], device='cuda:0') 0\n",
      "tensor([-30.1332, -43.5092, -11.9502, -34.7966], device='cuda:0') 2\n",
      "tensor([-33.0550,  -2.5587, -25.6633, -30.1752], device='cuda:0') 1\n",
      "tensor([-27.4397, -21.7965, -14.7708, -27.3431], device='cuda:0') 2\n",
      "tensor([-14.3248,  -5.8076,  -0.6996,  -4.9483], device='cuda:0') 2\n",
      "tensor([ -4.6808,  -6.0326,  -7.7552, -18.5542], device='cuda:0') 0\n",
      "tensor([-22.7721, -14.4457,  -0.3884, -14.4455], device='cuda:0') 2\n",
      "tensor([ -4.5938, -49.6452,  -9.1829,  -0.9415], device='cuda:0') 3\n",
      "tensor([-12.3493, -16.2650, -10.2156, -25.7712], device='cuda:0') 2\n",
      "Val FITB step [170/300] accuracy: 0.2603550295857988\n",
      "tensor([-19.7488,  -0.5732, -13.1963, -11.5647], device='cuda:0') 1\n",
      "tensor([ -1.5754, -48.2772, -32.2189, -17.8680], device='cuda:0') 0\n",
      "tensor([-4.7647e+01, -3.4637e+01, -9.3079e-04, -6.2103e+01], device='cuda:0') 2\n",
      "tensor([-26.2410, -57.1877,  -4.5048, -28.0603], device='cuda:0') 2\n",
      "tensor([ -8.8791, -19.5380, -23.5289,  -9.4360], device='cuda:0') 0\n",
      "tensor([-31.7482,  -8.4075,  -2.9414, -43.2128], device='cuda:0') 2\n",
      "forward 2\n",
      "forward 2\n",
      "Val FITB step [180/300] accuracy: 0.26256983240223464\n",
      "tensor([ -2.3502, -12.6039, -29.6728,  -3.6719], device='cuda:0') 0\n",
      "tensor([-20.0596,  -0.1430, -19.7565, -15.6611], device='cuda:0') 1\n",
      "tensor([-31.1954, -14.5449, -17.2496,  -0.4907], device='cuda:0') 3\n",
      "tensor([ -1.3165, -24.4909, -23.3699, -22.9280], device='cuda:0') 0\n",
      "tensor([-27.3257, -18.1125, -23.2733,  -3.5617], device='cuda:0') 3\n",
      "tensor([ -0.0110, -27.3493, -40.7288, -14.1852], device='cuda:0') 0\n",
      "tensor([-17.3950, -14.9741, -21.9239, -11.0852], device='cuda:0') 3\n",
      "tensor([ -4.8073, -18.7072, -19.6514,  -4.2874], device='cuda:0') 3\n",
      "Val FITB step [190/300] accuracy: 0.2698412698412698\n",
      "forward 2\n",
      "tensor([ -9.3764, -19.2410, -10.4208, -15.0707], device='cuda:0') 0\n",
      "forward 1\n",
      "forward 0\n",
      "forward 1\n",
      "tensor([-51.4554, -11.1688, -36.6859, -17.5996], device='cuda:0') 1\n",
      "tensor([-24.0114, -11.4200, -13.7286, -46.0277], device='cuda:0') 1\n",
      "tensor([-50.2520, -10.7803, -41.3124, -19.9292], device='cuda:0') 1\n",
      "tensor([-18.6101, -23.3830, -31.1280, -37.1036], device='cuda:0') 0\n",
      "Val FITB step [200/300] accuracy: 0.271356783919598\n",
      "forward 0\n",
      "tensor([-12.3919, -18.3631, -10.2278, -30.9470], device='cuda:0') 2\n",
      "tensor([-37.9903, -41.6676,  -6.7499,  -2.9065], device='cuda:0') 3\n",
      "tensor([-43.4364, -36.7094, -30.4546,  -0.0212], device='cuda:0') 3\n",
      "tensor([ -1.8157,  -5.2008, -27.3702, -33.3159], device='cuda:0') 0\n",
      "forward 1\n",
      "tensor([-77.2751, -12.4410, -33.5486, -38.8881], device='cuda:0') 1\n",
      "tensor([-11.3104, -23.9860, -20.2292, -30.9514], device='cuda:0') 0\n",
      "tensor([-24.5506, -19.3411,  -5.2304, -25.7290], device='cuda:0') 2\n",
      "tensor([ -8.9832, -36.0061, -17.2566, -31.7014], device='cuda:0') 0\n",
      "Val FITB step [210/300] accuracy: 0.27751196172248804\n",
      "tensor([-19.6969, -13.6012, -19.3928, -27.5366], device='cuda:0') 1\n",
      "tensor([-40.8503,  -2.6235, -15.8223, -30.4747], device='cuda:0') 1\n",
      "tensor([-16.0301, -10.8836, -16.5943,  -2.7142], device='cuda:0') 3\n",
      "tensor([-42.1313, -15.8523, -22.4942,  -1.3973], device='cuda:0') 3\n",
      "tensor([ -0.0268, -12.1234, -19.7891, -28.2527], device='cuda:0') 0\n",
      "tensor([-12.9042, -10.5684, -10.0390, -47.6122], device='cuda:0') 2\n",
      "tensor([-43.4490, -16.8540, -32.6729, -25.3988], device='cuda:0') 1\n",
      "Val FITB step [220/300] accuracy: 0.2694063926940639\n",
      "tensor([-21.0783, -56.6434,  -0.0056, -38.1979], device='cuda:0') 2\n",
      "forward 2\n",
      "tensor([-24.2696, -14.6194, -51.9928,  -5.1553], device='cuda:0') 3\n",
      "tensor([-15.4436, -29.1681, -15.5143,  -5.7013], device='cuda:0') 3\n",
      "tensor([-18.2479, -18.2006,  -0.0014, -15.7376], device='cuda:0') 2\n",
      "tensor([-22.7105, -61.3120,  -0.3770, -45.8510], device='cuda:0') 2\n",
      "forward 0\n",
      "tensor([-32.7681, -23.7793,  -6.6648,  -9.2344], device='cuda:0') 2\n",
      "tensor([ -3.6885, -21.0344,  -2.3419, -14.4854], device='cuda:0') 2\n",
      "Val FITB step [230/300] accuracy: 0.2663755458515284\n",
      "forward 2\n",
      "tensor([-12.0765, -20.4183, -26.4204,  -6.7376], device='cuda:0') 3\n",
      "tensor([-20.9456,  -0.0044, -38.5306, -24.4300], device='cuda:0') 1\n",
      "tensor([-3.6398e+01, -5.0898e+01, -4.1553e+01, -1.9073e-06], device='cuda:0') 3\n",
      "tensor([ -8.9948, -17.8521, -14.5195, -12.3513], device='cuda:0') 0\n",
      "tensor([ -0.0129, -10.3765, -53.6983, -51.6700], device='cuda:0') 0\n",
      "tensor([ -5.5170, -25.0906, -24.5726, -13.1266], device='cuda:0') 0\n",
      "tensor([ -4.5144, -24.6150, -14.7738,  -8.1145], device='cuda:0') 0\n",
      "tensor([-36.6339, -32.9546, -36.8160,  -8.5731], device='cuda:0') 3\n",
      "Val FITB step [240/300] accuracy: 0.27615062761506276\n",
      "tensor([-19.2509, -19.3347,  -0.2894, -48.7128], device='cuda:0') 2\n",
      "tensor([-18.8148, -49.1056,  -1.1220,  -9.2829], device='cuda:0') 2\n",
      "forward 2\n",
      "tensor([ -8.7776, -23.5639, -21.7532, -21.0748], device='cuda:0') 0\n",
      "tensor([-10.3670, -25.1085, -15.0788,  -5.0665], device='cuda:0') 3\n",
      "tensor([-20.8772, -34.0853, -16.8261,  -9.3919], device='cuda:0') 3\n",
      "tensor([-32.8305,  -6.3728,  -5.7057, -21.2979], device='cuda:0') 2\n",
      "Val FITB step [250/300] accuracy: 0.27309236947791166\n",
      "tensor([-15.9502, -31.5751, -25.6514, -29.2505], device='cuda:0') 0\n",
      "tensor([-15.2846, -23.7877,  -0.0090, -16.8028], device='cuda:0') 2\n",
      "tensor([-20.7403, -32.2598, -10.8989, -39.6986], device='cuda:0') 2\n",
      "tensor([-27.7957, -30.5938, -18.6303,  -8.8569], device='cuda:0') 3\n",
      "tensor([-12.4709,  -6.9106, -43.6966,  -3.8961], device='cuda:0') 3\n",
      "tensor([-3.1955e+01, -3.1291e+01, -2.5742e+01, -5.7220e-06], device='cuda:0') 3\n",
      "tensor([-11.9661,  -4.1207,  -1.9868, -28.7717], device='cuda:0') 2\n",
      "tensor([-33.4618, -48.0665,  -0.0317, -11.7569], device='cuda:0') 2\n",
      "tensor([-17.6247, -20.7477, -34.8231, -41.5684], device='cuda:0') 0\n",
      "tensor([-10.2060,  -3.2057,  -9.5300,  -8.9449], device='cuda:0') 1\n",
      "Val FITB step [260/300] accuracy: 0.2702702702702703\n",
      "forward 3\n",
      "tensor([-20.0903, -11.8470,  -1.5892,  -6.8652], device='cuda:0') 2\n",
      "tensor([ -0.0201, -28.6707, -10.5220, -31.4735], device='cuda:0') 0\n",
      "tensor([ -8.8964, -51.8773, -10.2220, -31.7916], device='cuda:0') 0\n",
      "tensor([-18.7388, -11.7526, -22.3627,  -0.0649], device='cuda:0') 3\n",
      "tensor([ -2.2854, -11.7685, -62.8765, -25.4923], device='cuda:0') 0\n",
      "forward 3\n",
      "tensor([-14.0509, -12.4555, -23.2686,  -5.3832], device='cuda:0') 3\n",
      "tensor([-24.9510, -20.3287,  -4.4489, -17.5122], device='cuda:0') 2\n",
      "tensor([ -9.8530,  -8.0575,  -2.3899, -19.4601], device='cuda:0') 2\n",
      "Val FITB step [270/300] accuracy: 0.27137546468401486\n",
      "forward 1\n",
      "tensor([-17.2351,  -2.3017, -30.6511, -25.5140], device='cuda:0') 1\n",
      "tensor([-12.9862, -35.7717, -21.1541,  -5.0881], device='cuda:0') 3\n",
      "tensor([ -8.6713, -19.3507, -41.9078, -35.4619], device='cuda:0') 0\n",
      "tensor([-23.1803, -24.7997,  -7.1758,  -7.5260], device='cuda:0') 2\n",
      "tensor([-44.4867, -22.3260, -15.8621, -21.7365], device='cuda:0') 2\n",
      "tensor([-12.3891, -19.8210,  -6.0302, -50.9981], device='cuda:0') 2\n",
      "tensor([-48.1850, -31.4722, -32.5613, -17.5422], device='cuda:0') 3\n",
      "Val FITB step [280/300] accuracy: 0.26523297491039427\n",
      "tensor([ -3.9544,  -5.3441,  -6.1852, -12.9359], device='cuda:0') 0\n",
      "tensor([-50.6274, -43.3590,  -5.2614, -28.0894], device='cuda:0') 2\n",
      "tensor([ -0.6966, -18.9364,  -8.9849, -24.3368], device='cuda:0') 0\n",
      "tensor([-24.3115, -33.6869,  -0.0136, -27.0985], device='cuda:0') 2\n",
      "tensor([-31.0507,  -0.1680, -39.2353, -42.4874], device='cuda:0') 1\n",
      "tensor([-14.6907, -15.4403, -32.8686, -28.0936], device='cuda:0') 0\n",
      "tensor([-29.3085,  -0.0098, -28.4427, -44.6322], device='cuda:0') 1\n",
      "tensor([ -5.0429, -21.7407, -31.2869, -10.9105], device='cuda:0') 0\n",
      "Val FITB step [290/300] accuracy: 0.2698961937716263\n",
      "tensor([-29.3313, -15.9603, -23.5167, -19.5105], device='cuda:0') 1\n",
      "forward 1\n",
      "tensor([-33.8699,  -6.8548, -26.3024, -36.8428], device='cuda:0') 1\n",
      "tensor([-26.0797,  -0.4398, -39.0812,  -8.9174], device='cuda:0') 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([ -8.3880, -27.4114, -21.1853, -47.3962], device='cuda:0') 0\n",
      "tensor([ -9.9798, -31.3001, -20.5174, -17.4825], device='cuda:0') 0\n",
      "Val FITB step [300/300] accuracy: 0.26755852842809363\n",
      "FITB accracy 0.26666666666666666\n"
     ]
    }
   ],
   "source": [
    "cnt = 0\n",
    "correct = 0\n",
    "for i in range(len(fitb_dataset)):\n",
    "    if (i + 1) % 10 == 0:\n",
    "        print('Val FITB step [{}/{}] accuracy: {}'.format(i + 1, len(fitb_dataset), correct / cnt))\n",
    "    sample = fitb_dataset[i]\n",
    "    question_nums = sample['images'].shape[0]\n",
    "    for key in sample:\n",
    "        if isinstance(sample[key], torch.Tensor):\n",
    "            sample[key] = sample[key].to(const.device)\n",
    "\n",
    "    # [num, emb_size]\n",
    "    question_embeddings = self.im2rnn_embedding(sample['images'].unsqueeze(0))\n",
    "    answer_embeddings = self.im2rnn_embedding(sample['answer_images'].unsqueeze(0))\n",
    "\n",
    "    # blank_position严格从1开始数\n",
    "    blank_position = sample['blank_position']\n",
    "    if blank_position == question_nums + 1:\n",
    "        f_lstm_hidden = self.f_lstm(question_embeddings)[0]\n",
    "        f_lstm_pred = self.f_lstm_pred(f_lstm_hidden[:, question_nums - 1, :])\n",
    "        f_scores = answer_embeddings.squeeze(0).matmul(f_lstm_pred.transpose(0, 1))\n",
    "        f_scores = F.log_softmax(f_scores, dim=0)\n",
    "        ans = f_scores.argmax().item()\n",
    "#         print(\"forward\", ans)\n",
    "    elif blank_position == 1:\n",
    "        b_question_embeddings = flip(question_embeddings, 1)\n",
    "        b_lstm_hidden = self.b_lstm(b_question_embeddings)[0]\n",
    "        b_lstm_pred = self.b_lstm_pred(b_lstm_hidden[:, question_nums - 1, :])\n",
    "        b_scores = answer_embeddings.squeeze(0).matmul(b_lstm_pred.transpose(0, 1))\n",
    "        b_scores = F.log_softmax(b_scores, dim=0)\n",
    "        ans = b_scores.argmax().item()\n",
    "#         print(\"backward\", ans)\n",
    "    else:\n",
    "        f_lstm_hidden = self.f_lstm(question_embeddings)[0]\n",
    "        f_lstm_pred = self.f_lstm_pred(f_lstm_hidden[:, blank_position - 2, :])\n",
    "        f_scores = answer_embeddings.squeeze(0).matmul(f_lstm_pred.transpose(0, 1))\n",
    "        f_scores = F.log_softmax(f_scores, dim=0)\n",
    "\n",
    "        b_question_embeddings = flip(question_embeddings, 1)\n",
    "        b_blank_position = question_nums - blank_position + 1 # 反转\n",
    "\n",
    "        b_lstm_hidden = self.b_lstm(b_question_embeddings)[0]\n",
    "        b_lstm_pred = self.b_lstm_pred(b_lstm_hidden[:, b_blank_position - 2, :])\n",
    "        b_scores = answer_embeddings.squeeze(0).matmul(b_lstm_pred.transpose(0, 1))\n",
    "        b_scores = F.log_softmax(b_scores, dim=0)\n",
    "\n",
    "        scores = f_scores * const.F_WEIGHT_SPARSE_SOFTMAX + b_scores * const.B_WEIGHT_SPARSE_SOFTMAX\n",
    "        ans = scores.argmax().item()\n",
    "#         print(scores.reshape(-1), ans)\n",
    "    if ans == 0:\n",
    "        correct += 1\n",
    "    cnt += 1\n",
    "print('FITB accracy {}'.format(correct / cnt))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
